This notebook is written in Python. It makes use of the features computed from the Torch notebook examples_features.ipynb. An image is passed through Context Encoder. We retain the 'code', that is the 4000 features at the interface of the encoder and the decoder. In the paper, the authors use these features for segmentation and classification, with the small difference that they passed the images through a Context Encoder trained for random block inpainting, a network that is not available from their GitHub repository. Hopefully the features learned with the central block inpainting are also useful, but we'll see that often when the center of the image is occluded, most of the information about the main object of the image is basically removed. This notebook merges the features computed from single images into dataframes from efficiency and convenience. Then these features are passed through t-SNE with 2 components so that we can visualize the high-dimensional features. The figures obtained are quite interesting and are discussed in the report.
In [1]:
import pandas as pd
import numpy as np
import matplotlib.pylab as plt
%matplotlib inline
from sklearn.decomposition import PCA
import pickle
from time import time
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import offsetbox
from sklearn import (manifold, datasets, decomposition, ensemble,
discriminant_analysis, random_projection)
from skimage.io import imread
from skimage.transform import rescale
from skimage.color import rgb2gray
# pickle.load(open('id_to_filename', 'rb'))
# pickle.load(open('filename_to_id', 'rb'))
In [2]:
# combine the features of each image into one dataframe
#
# features = []
# for i in range(112):
# filename = id_to_filename[str(i)]
# if 'imagenet' in filename:
# feature = list(pd.read_csv('features/imagenet/' + filename + '.csv', header=None).values[0])
# feature.append(0) # tag for imagenet
# features.append(feature)
# elif 'paris' in filename:
# feature = list(pd.read_csv('features/paris/' + filename + '.csv', header=None).values[0])
# feature.append(1) # tag for paris
# features.append(feature)
# elif 'textures' in filename:
# feature = list(pd.read_csv('features/textures/' + filename + '.csv', header=None).values[0])
# feature.append(2) # tag for textures
# features.append(feature)
# elif 'mires' in filename:
# feature = list(pd.read_csv('features/mires/' + filename + '.csv', header=None).values[0])
# feature.append(3) # tag for mires
# features.append(feature)
# # for idx, f in enumerate(features):
# # features[idx] = f[0]
# features = np.array(features)
# features.shape
# features = pd.DataFrame(features)
# features = pd.DataFrame(features)
# features.to_csv('features.csv', index=False, header=False)
features = pd.read_csv('features.csv', header=None).values
PCA¶
In [140]:
features = pd.read_csv('features.csv', header=None).values
pca = PCA(n_components=2)
pca.fit(features[:, :4000])
X_pca = pca.transform(features[:, :4000])
fig, ax = plt.subplots(figsize=(20, 10))
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=features[:, 4000])
plt.colorbar()
plt.title('PCA\n yellow: mires, green: textures, blue: paris, purple: imagenet')
plt.tight_layout()
plt.savefig('pca.png')
t-SNE¶
In [96]:
features = pd.read_csv('features.csv', header=None).values
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
X = features[:, :4000]
X_tsne = tsne.fit_transform(X)
fig, ax = plt.subplots(figsize=(20, 10))
plt.scatter(X_tsne[:, 0], X_tsne[:, 1], c=features[:, 4000])
plt.colorbar()
plt.title('t-SNE\n yellow: mires, green: textures, blue: paris, purple: imagenet')
plt.tight_layout()
plt.savefig('tsne.png')
Plot embedding¶
In [130]:
num_files = 112
id_to_filename = pickle.load(open('id_to_filename', 'rb'))
images = []
for i in range(num_files):
filename = id_to_filename[str(i)]
if 'imagenet' in filename:
im = imread('images/imagenet/' + filename + '.png')
images.append(rescale(im, [0.25, 0.25]))
elif 'paris' in filename:
im = imread('images/paris/' + filename + '.png')
images.append(rescale(im, [0.25, 0.25]))
elif 'textures' in filename:
im = imread('images/textures/' + filename + '.png')
images.append(rescale(im, [0.25, 0.25]))
elif 'mires' in filename:
im = imread('images/mires/' + filename + '.png')
images.append(rescale(im, [0.25, 0.25]))
X = np.array(images)
y = features[:, 4000]
name = {'0': 'imnet', '1': 'paris', '2': 'textures', '3': 'mires'}
In [131]:
#----------------------------------------------------------------------
# Plot thumbnails
plt.figure(figsize=(50, 25))
n_img_per_row = 10
img = np.ones((34 * n_img_per_row, 34 * n_img_per_row, 3))
Y = np.copy(X)
np.random.shuffle(Y)
for i in range(n_img_per_row):
ix = 34 * i + 1
for j in range(n_img_per_row):
iy = 34 * j + 1
# img[ix:ix + 32, iy:iy + 32, :] = rescale(X[i * n_img_per_row + j], [0.25, 0.25])
img[ix:ix + 32, iy:iy + 32, :] = Y[i * n_img_per_row + j]
plt.imshow(img)
plt.xticks([])
plt.yticks([])
plt.title('A selection from the imagenet, paris, textures and mires datasets')
plt.tight_layout()
plt.savefig('thumbnail.png')
In [132]:
y = features[:, 4000]
name = {'0': 'imnet', '1': 'paris', '2': 'textures', '3': 'mires'}
In [133]:
# borrowed from http://scikit-learn.org/stable/auto_examples/manifold/plot_lle_digits.html
# Scale and visualize the embedding vectors
def plot_embedding(X_tsne, title=None, text=False, epsilon=4e-3, figsize=(50, 25)):
x_min, x_max = np.min(X_tsne, 0), np.max(X_tsne, 0)
X_tsne = (X_tsne - x_min) / (x_max - x_min)
fig, ax = plt.subplots(figsize=figsize)
if text:
for i in range(X.shape[0]):
plt.text(X_tsne[i, 0]+.01, X_tsne[i, 1]+.01, name[str(int(y[i]))],
color=plt.cm.Set1(y[i] / 10.),
fontdict={'weight': 'bold', 'size': 9})
if hasattr(offsetbox, 'AnnotationBbox'):
# only print thumbnails with matplotlib > 1.0
shown_images = np.array([[1., 1.]]) # just something big
for i in range(X_tsne.shape[0]):
dist = np.sum((X_tsne[i] - shown_images) ** 2, 1)
if np.min(dist) < epsilon:
# don't show points that are too close
continue
shown_images = np.r_[shown_images, [X_tsne[i]]]
imagebox = offsetbox.AnnotationBbox(
offsetbox.OffsetImage(X[i], cmap=plt.cm.gray_r),
X_tsne[i])
ax.add_artist(imagebox)
plt.xticks([]), plt.yticks([])
if title is not None:
plt.title(title)
In [141]:
plot_embedding(X_tsne, epsilon=4e-5, figsize=(25, 10), text=True)
# plt.savefig('viz_tsne.pdf')
In [143]:
plot_embedding(X_pca, epsilon=4e-5, figsize=(25, 10), text=True)
# plt.savefig('viz_pca.pdf')
few images from recvis/assignment2¶
In [146]:
# features = []
# num_images = 2113
# for i in range(1, num_images + 1):
# filename = str(i)
# feature = list(pd.read_csv('classifier/features/' + filename + '.csv', header=None).values[0])
# features.append(feature)
# features = np.array(features)
# features.shape
# features = pd.DataFrame(features)
# features = pd.DataFrame(features)
# features.to_csv('classifier/features.csv', index=False, header=False)
features = pd.read_csv('classifier/features.csv', header=None).values
In [152]:
images = []
for i in range(1, num_images + 1):
filename = str(i)
im = imread('classifier/images/' + filename + '_im.png')
images.append(rescale(im, [0.25, 0.25]))
X = np.array(images)
In [150]:
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
X_tsne = tsne.fit_transform(features)
fig, ax = plt.subplots(figsize=(20, 10))
plt.scatter(X_tsne[:, 0], X_tsne[:, 1])
plt.tight_layout()
In [149]:
plt.scatter(X_tsne[:, 0], X_tsne[:, 1])
plt.tight_layout()
In [11]:
y = np.zeros(X_tsne.shape[0])
name = {'0': 'imnet', '1': 'paris', '2': 'textures', '3': 'mires'}
In [155]:
plot_embedding(X_tsne, figsize=(100, 50), text=False, epsilon=4e-5)
# plt.savefig('embedding.pdf')
